package cn.dawn.webcollector.crawler;

import cn.dawn.webcollector.extract.Extractor;
import cn.dawn.webcollector.extract.ExtractorParams;
import cn.dawn.webcollector.extract.Extractors;
import cn.dawn.webcollector.extract.RegexExtractorFactory;
import cn.dawn.webcollector.model.Links;
import cn.dawn.webcollector.model.Page;

public class MultiExtractorCrawler extends BreadthCrawler {

	protected RegexExtractorFactory regexExtractorFactory = new RegexExtractorFactory();

	public MultiExtractorCrawler(String crawlPath, boolean autoParse) {
		super(crawlPath, autoParse);
	}

	@Override
	public void visit(Page page, Links nextLinks) {
		try {
			Extractors extractors = regexExtractorFactory.createExtractor(page);
			for (Extractor extractor : extractors) {
				try {
					extractor.execute(nextLinks);
				} catch (Exception ex) {
					LOG.info("Exception", ex);
				}
			}
		} catch (Exception ex) {
			LOG.info("Exception", ex);
		}
	}

	public void addExtractor(String urlRegex, Class<? extends Extractor> extractorClass) {
		regexExtractorFactory.addExtractor(urlRegex, extractorClass);
	}

	public void addExtractor(String urlRegex, Class<? extends Extractor> extractorClass, ExtractorParams params) {
		regexExtractorFactory.addExtractor(urlRegex, extractorClass, params);
	}

	public RegexExtractorFactory getRegexExtractorFactory() {
		return regexExtractorFactory;
	}

	public void setRegexExtractorFactory(RegexExtractorFactory regexExtractorFactory) {
		this.regexExtractorFactory = regexExtractorFactory;
	}

}
